graph drop _all
global mainpath ".."

***********************************************************************
*load data from individual Excel and csv files
***********************************************************************
*fed funds rate
import delimited "$mainpath\data\FEDFUNDS.csv", varnames(1) clear
gen d=date(date,"YMD")
gen month = mofd(d)
format month %tm
keep month fedfunds
tempfile ff
save `ff', replace

*baseline gamma
import delimited "$mainpath\output\bluechip_rule_regressions.csv", varnames(1) clear 

rename gamma_fe gamma_ffe
gen DATE=date(date,"YMD")
format DATE %td
gen month=mofd(DATE)
format month %tm
format DATE %td
drop if month==.
gsort month
tsset month
save "$mainpath\tmp\policyrule_monthly_regressions.dta", replace

*inertial gamma
import delimited "$mainpath\output\bluechip_rule_inertial.csv", clear
rename gamma gamma_inertia
rename beta beta_inertia
rename gamma_ub gamma_inertia_ub
rename gamma_lb gamma_inertia_lb
rename beta_ub beta_inertia_ub
rename beta_lb beta_inertia_lb

rename rho rho
gen DATE=date(date,"YMD")
gen month=mofd(DATE)
format month %tm
format DATE %td
drop if month==.
gsort month
tsset month

keep month DATE gamma_inertia* beta_inertia* rho 

save "$mainpath\tmp\policyrule_monthly_inertial.dta", replace

*load CFNAI
import excel "$mainpath\data\CFNAIMA3.xls", sheet("FRED Graph") clear firstrow
rename observation* DATE
gen month=mofd(DATE)
format month %tm
drop if month==.
gsort month 
tsset month
save "$mainpath\tmp\CFNAI.dta", replace


*load GSW yields for full sample 
import delimited "$mainpath\data\feds200628.csv", varnames(10) stringcols(1)  case(preserve) clear
rename Date DATE
keep DATE
gen t=_n
gsort t
save "$mainpath\tmp\gsw_date.dta", replace
import delimited "$mainpath\data\feds200628.csv", varnames(10) numericcols(_all) case(preserve) clear
drop Date
gen t=_n
gsort t
merge 1:1 t using "$mainpath\tmp\gsw_date.dta"
drop _merge
drop t
gen date=date(DATE,"YMD")
drop DATE
rename date Date
gen month=mofd(Date)
format month %tm
drop if month==. 
drop if SVENY10==.
gsort month -Date
by month: keep if _n==1
gsort month 
save "$mainpath\tmp\GSW_nominal_monthend.dta", replace

*compute mean interest rate forecasts from individual BC data
use "$mainpath\data\bc_May23.dta", clear
gen ym = mofd(date)
rename y1 y1_
rename y2 y2_
rename y5 y5_
rename y10 y10_

collapse (mean) y1_ y2_ y5_ y10_ gap ff, by(ym h)
reshape wide y1_ y2_ y5_ y10_ gap ff, i(ym) j(h)

rename ym month
gsort month
save "$mainpath\tmp\bc_timeseries.dta", replace

*compute disagreement from individual BC data
use "$mainpath\data\bc_May23.dta", clear
gen ym=mofd(date)

keep if h==4

tempfile coeffs_pctile_ff
statsby r(r1) r(r2), by(ym) saving(`coeffs_pctile_ff', replace): _pctile ff, percentiles(10 90)

tempfile coeffs_pctile_y2
statsby r(r1) r(r2), by(ym) saving(`coeffs_pctile_y2', replace): _pctile y2, percentiles(10 90)

tempfile coeffs_pctile_y10
statsby r(r1) r(r2), by(ym) saving(`coeffs_pctile_y10', replace): _pctile y10, percentiles(10 90)

use `coeffs_pctile_ff', clear
rename _stat_1 ff_p10
rename _stat_2 ff_p90
gen ff_9010=ff_p90-ff_p10
lab var ff_9010 "90-10 Fed Funds"

merge 1:1 ym using `coeffs_pctile_y2'
drop _merge
rename _stat_1 y2_p10
rename _stat_2 y2_p90
gen y2_9010=y2_p90-y2_p10
lab var y2_9010 "90-10 Y2"

merge 1:1 ym using `coeffs_pctile_y10'
drop _merge
rename _stat_1 y10_p10
rename _stat_2 y10_p90
gen y10_9010=y10_p90-y10_p10
lab var y10_9010 "90-10 Y10"

gen month=ym
format ym %tm
gsort ym
keep month ym ff_9010 y2_9010 y10_9010 
save "$mainpath\tmp\disagreement_ff.dta", replace

**merge
use "$mainpath\tmp\policyrule_monthly_regressions.dta", clear

merge 1:1 month using "$mainpath\tmp\policyrule_monthly_inertial.dta"
drop _merge

merge 1:1 month using "$mainpath\tmp\GSW_nominal_monthend.dta"
drop _merge

merge 1:1 month using "$mainpath\tmp\CFNAI.dta"
drop _merge

merge 1:1 month using "$mainpath\tmp\bc_timeseries.dta"
drop _merge

merge 1:1 month using "$mainpath\tmp\disagreement_ff.dta"
drop _merge

merge 1:1 month using `ff'
drop _merge

order month DATE

*load bond return second moments
gen q=qofd(DATE)
gsort q month

save "$mainpath\tmp\policyrule_merged_June2023.dta", replace


**analyze
use "$mainpath\tmp\policyrule_merged_June2023.dta", clear
***********************************************
*define variables
***********************************************
drop if month==.
gsort month
tsset month
gen year=yofd(Date)

*one-year realized excess return on five-year bond
gen xr_nom5yr_1yr=5*L12.SVENY05-4*SVENY04-L12.SVENY01
*2-year excess return
gen xr_nom5yr_2yr=5*L24.SVENY05-3*SVENY03-2*L24.SVENY02

*one-year subjective risk premium
gen dur6=(1-(1+SVENPY06/100)^(-6))/(1-(1+SVENPY06/100)^(-1))
gen dur11=(1-(1+SVENPY11/100)^(-11))/(1-(1+SVENPY11/100)^(-1))
gen xr_nom5yr_survey=dur6*SVENPY06-(dur6-1)*f.y5_4-SVENY01
gen xr_nom10yr_survey=dur11*SVENPY11-(dur11-1)*f.y10_4-SVENY01

*principal components
pca SVENY01 SVENY02 SVENY03 SVENY05 SVENY07 SVENY10 SVENY15 SVENY20, comp(3)
predict pc1 pc2 pc3

gen TERM=SVENY10-SVENY01
gen TERM5=SVENY05-SVENY01

*forecast errors 
gen fe4 = fedfunds-l12.ff4
gen fe2 = fedfunds-l6.ff2
replace fe4=f.fe4
replace fe2=f.fe2

gen gamma_zscore=0
gen gamma_X_CFNAI=0
gen fgamma_X_CFNAI=0
gen l5gamma_X_CFNAI=0
gen l11gamma_X_CFNAI=0
gen gamma=0


gen t=_n
gsort t
tsset t


*****************************************************************************
*loop over versions of gamma
*****************************************************************************
egen rho_zscore=std(rho)
egen beta_inertia_zscore=std(beta_inertia)
egen gamma_inertia_zscore=std(gamma_inertia)
egen CFNAI_zscore=std(CFNAI)

foreach var of varlist gamma_ffe gamma_inertia{

replace gamma=`var'

*standardize gamma
sum gamma 
drop gamma_zscore
egen gamma_zscore = std(gamma)
*uncomment this next line for results without standardization
replace gamma_zscore=gamma
replace gamma_X_CFNAI=gamma_zscore*CFNAI_zscore
replace fgamma_X_CFNAI=f.gamma_zscore*CFNAI_zscore
replace l5gamma_X_CFNAI=L11.gamma_zscore*CFNAI_zscore
replace l11gamma_X_CFNAI=L5.gamma_zscore*CFNAI_zscore


*Table 3 in the paper
ivreg2 xr_nom5yr_survey f.gamma_zscore, kernel(bartlett) bw(auto) robust
estimates store sub1
ivreg2 xr_nom5yr_survey f.gamma_zscore TERM, kernel(bartlett) bw(auto) robust
estimates store sub2
ivreg2 xr_nom5yr_survey f.gamma_zscore pc1 pc2 pc3, kernel(bartlett) bw(auto) robust
estimates store sub3

ivreg2 xr_nom10yr_survey f.gamma_zscore, kernel(bartlett) bw(auto) robust
estimates store sub4
ivreg2 xr_nom10yr_survey f.gamma_zscore TERM, kernel(bartlett) bw(auto) robust
estimates store sub5
ivreg2 xr_nom10yr_survey f.gamma_zscore pc1 pc2 pc3, kernel(bartlett) bw(auto) robust
estimates store sub6

esttab sub1 sub2 sub3 sub4 sub5 sub6 using "$mainpath/tables/returns_surveys_`var'.csv", star(* .10 ** .05 *** .01)  b(a2) nomtitle replace r2(2) se(a2)

*predict realized excess returns for pre-Covid sample ending Feb 2020
ivreg2 xr_nom5yr_1yr L12.pc1 L12.pc2 L12.pc3  L12.gamma_zscore if L12.xr_nom5yr_survey~=.&month<=721, kernel(bartlett) bw(17) robust
estimates store returns1
ivreg2 xr_nom5yr_1yr L12.pc1 L12.pc2 L12.pc3  L12.gamma_zscore L12.CFNAI_zscore if L12.xr_nom5yr_survey~=.&month<=721, kernel(bartlett) bw(17) robust
estimates store returns2
ivreg2 xr_nom5yr_1yr L12.pc1 L12.pc2 L12.pc3  L12.gamma_zscore L12.CFNAI_zscore L12.gamma_X_CFNAI if L12.xr_nom5yr_survey~=.&month<=721, kernel(bartlett) bw(17) robust
estimates store returns3
esttab returns1 returns2 returns3 using "$mainpath/tables/returns_1yr_5yr_controls_`var'.csv", star(* .10 ** .05 *** .01)  b(a2) nomtitle replace r2(2)

ivreg2 xr_nom5yr_2yr L24.pc1 L24.pc2 L24.pc3  L24.gamma_zscore if L24.xr_nom5yr_survey~=.&month<=721, kernel(bartlett) bw(35) robust
estimates store returns1
ivreg2 xr_nom5yr_2yr L24.pc1 L24.pc2 L24.pc3  L24.gamma_zscore L24.CFNAI_zscore if L24.xr_nom5yr_survey~=.&month<=721, kernel(bartlett) bw(35) robust
estimates store returns2
ivreg2 xr_nom5yr_2yr L24.pc1 L24.pc2 L24.pc3  L24.gamma_zscore L24.CFNAI_zscore gamma_X_CFNAI if L24.xr_nom5yr_survey~=.&month<=721, kernel(bartlett) bw(35) robust
estimates store returns3
esttab returns1 returns2 returns3 using "$mainpath/tables/returns_2yr_5yr_controls_`var'.csv", star(* .10 ** .05 *** .01)  b(a2) nomtitle replace r2(2)

*Table 3 controlling for disagreement 
ivreg2 xr_nom5yr_survey f.gamma_zscore f.ff_9010 pc1 pc2 pc3, kernel(bartlett) bw(auto) robust
estimates store sub2
ivreg2 xr_nom5yr_survey f.gamma_zscore f.y2_9010 pc1 pc2 pc3, kernel(bartlett) bw(auto) robust
estimates store sub3
ivreg2 xr_nom5yr_survey f.gamma_zscore f.y10_9010 pc1 pc2 pc3, kernel(bartlett) bw(auto) robust
estimates store sub4

ivreg2 xr_nom10yr_survey f.gamma_zscore f.ff_9010 pc1 pc2 pc3, kernel(bartlett) bw(auto) robust
estimates store sub6
ivreg2 xr_nom10yr_survey f.gamma_zscore f.y2_9010 pc1 pc2 pc3, kernel(bartlett) bw(auto) robust
estimates store sub7
ivreg2 xr_nom10yr_survey f.gamma_zscore f.y10_9010 pc1 pc2 pc3, kernel(bartlett) bw(auto) robust
estimates store sub8

esttab sub2 sub3 sub4 sub6 sub7 sub8 using "$mainpath/tables/returns_surveys_9010_`var'.csv", star(* .10 ** .05 *** .01)  b(a2) nomtitle replace r2(2) drop(pc*)


*fed funds forecast errors
ivreg2 fe4 L12.CFNAI_zscore L12.fedfunds if L11.gamma~=., kernel(bartlett) bw(18)
estimates store ff2
ivreg2 fe2 L6.CFNAI_zscore L6.fedfunds if L5.gamma~=., kernel(bartlett) bw(9)
estimates store ff1
ivreg2 fe4 L12.CFNAI_zscore L11.gamma L12.fgamma_X_CFNAI L12.fedfunds, kernel(bartlett) bw(18)
estimates store ff4
ivreg2 fe2 L6.CFNAI_zscore L5.gamma L6.fgamma_X_CFNAI L6.fedfunds, kernel(bartlett) bw(9)
estimates store ff3

esttab ff1 ff2 ff3 ff4 using "$mainpath/tables/forecast_errors_`var'.csv", star(* .10 ** .05 *** .01)  b(a2) nomtitle replace r2(2) se(a2)
}

*multivariate regressions for expected returns
ivreg2 xr_nom5yr_survey f.rho_zscore f.gamma_inertia_zscore f.beta_inertia_zscore, kernel(bartlett) bw(auto) robust
estimates store sub1
ivreg2 xr_nom5yr_survey f.rho_zscore f.gamma_inertia_zscore f.beta_inertia_zscore TERM, kernel(bartlett) bw(auto) robust
estimates store sub2
ivreg2 xr_nom5yr_survey f.rho_zscore f.gamma_inertia_zscore f.beta_inertia_zscore pc1 pc2 pc3, kernel(bartlett) bw(auto) robust
estimates store sub3

ivreg2 xr_nom10yr_survey f.rho_zscore f.gamma_inertia_zscore f.beta_inertia_zscore, kernel(bartlett) bw(auto) robust
estimates store sub4
ivreg2 xr_nom10yr_survey f.rho_zscore f.gamma_inertia_zscore  f.beta_inertia_zscore TERM, kernel(bartlett) bw(auto) robust
estimates store sub5
ivreg2 xr_nom10yr_survey f.rho_zscore f.gamma_inertia_zscore f.beta_inertia_zscore pc1 pc2 pc3, kernel(bartlett) bw(auto) robust
estimates store sub6

esttab sub1 sub2 sub3 sub4 sub5 sub6 using "$mainpath/tables/returns_surveys_multi.csv", star(* .10 ** .05 *** .01)  b(2) nomtitle replace r2(2) se(2) drop(pc*) 

*correlation gamma vs. disagreement
pwcorr gamma_ffe gamma_inertia ff_9010 y2_9010 y10_9010